{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载数据\n",
    "file_path_train = \"./data/NSL_KDD-master/KDDTrain+.csv\"\n",
    "file_path_test = \"./data/NSL_KDD-master/KDDTest+.csv\"\n",
    "train_data = pd.read_csv(file_path_train)\n",
    "test_data = pd.read_csv(file_path_test)\n",
    "data_columns =  [\"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\n",
    "            \"dst_bytes\",\"land_f\",\"wrong_fragment\",\"urgent\",\"hot\",\"num_failed_logins\",\n",
    "            \"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\n",
    "            \"num_file_creations\",\"num_shells\",\"num_access_files\",\"num_outbound_cmds\",\n",
    "            \"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\n",
    "            \"srv_serror_rate\",\"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\n",
    "            \"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\",\n",
    "            \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\n",
    "            \"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\"dst_host_srv_serror_rate\",\n",
    "            \"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\"labels\",\"dificulty\"]\n",
    "\n",
    "train_data.columns = data_columns\n",
    "test_data.columns = data_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land_f</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>labels</th>\n",
       "      <th>dificulty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>private</td>\n",
       "      <td>REJ</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>neptune</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>tcp</td>\n",
       "      <td>ftp_data</td>\n",
       "      <td>SF</td>\n",
       "      <td>12983</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>eco_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>saint</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>tcp</td>\n",
       "      <td>telnet</td>\n",
       "      <td>RSTO</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.71</td>\n",
       "      <td>mscan</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>267</td>\n",
       "      <td>14515</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>normal</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration protocol_type   service  flag  src_bytes  dst_bytes  land_f  \\\n",
       "0         0           tcp   private   REJ          0          0       0   \n",
       "1         2           tcp  ftp_data    SF      12983          0       0   \n",
       "2         0          icmp     eco_i    SF         20          0       0   \n",
       "3         1           tcp    telnet  RSTO          0         15       0   \n",
       "4         0           tcp      http    SF        267      14515       0   \n",
       "\n",
       "   wrong_fragment  urgent  hot  ...  dst_host_same_srv_rate  \\\n",
       "0               0       0    0  ...                    0.00   \n",
       "1               0       0    0  ...                    0.61   \n",
       "2               0       0    0  ...                    1.00   \n",
       "3               0       0    0  ...                    0.31   \n",
       "4               0       0    0  ...                    1.00   \n",
       "\n",
       "   dst_host_diff_srv_rate  dst_host_same_src_port_rate  \\\n",
       "0                    0.06                         0.00   \n",
       "1                    0.04                         0.61   \n",
       "2                    0.00                         1.00   \n",
       "3                    0.17                         0.03   \n",
       "4                    0.00                         0.01   \n",
       "\n",
       "   dst_host_srv_diff_host_rate  dst_host_serror_rate  \\\n",
       "0                         0.00                  0.00   \n",
       "1                         0.02                  0.00   \n",
       "2                         0.28                  0.00   \n",
       "3                         0.02                  0.00   \n",
       "4                         0.03                  0.01   \n",
       "\n",
       "   dst_host_srv_serror_rate  dst_host_rerror_rate  dst_host_srv_rerror_rate  \\\n",
       "0                       0.0                  1.00                      1.00   \n",
       "1                       0.0                  0.00                      0.00   \n",
       "2                       0.0                  0.00                      0.00   \n",
       "3                       0.0                  0.83                      0.71   \n",
       "4                       0.0                  0.00                      0.00   \n",
       "\n",
       "    labels  dificulty  \n",
       "0  neptune         21  \n",
       "1   normal         21  \n",
       "2    saint         15  \n",
       "3    mscan         11  \n",
       "4   normal         21  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 显示一部分数据\n",
    "train_data.head()\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 提取标签\n",
    "X_train = train_data.drop('labels',axis=1)\n",
    "X_train = X_train.drop('dificulty', axis=1)\n",
    "labels_train = train_data['labels']\n",
    "\n",
    "labels = labels_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 由于使用的是独热编码\n",
    "# 但是测试集中的 service 字段与训练集中的 service 缺失了7中数据因此将缺失的这七种数据加入测试集\n",
    "# 保证测试集 service 字段能够能够进行 one-hot 编码\n",
    "service = X_train['service']\n",
    "test_service = test_data['service'] \n",
    "different_service_type = np.array(list(set(service) - set(test_service)))\n",
    "np_service = np.array(list(service))\n",
    "miss_service_data_index = np.array([True if type_service in different_service_type else False for type_service in np_service])\n",
    "add_to_test_data = train_data.values[miss_service_data_index.nonzero()]\n",
    "add_to_test_data\n",
    "\n",
    "test_data = np.concatenate((test_data.values,add_to_test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 测试集和训练集统一特征\n",
    "test_data = pd.DataFrame(test_data)\n",
    "test_data.columns = data_columns\n",
    "\n",
    "X_test = test_data.drop('labels',axis=1)\n",
    "X_test = X_test.drop('dificulty', axis=1)\n",
    "labels_test = test_data['labels']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 提取特征方便进行 one-hot 编码\n",
    "test_protocol_type = X_test['protocol_type']\n",
    "test_service = X_test['service']\n",
    "test_flag = X_test['flag']\n",
    "\n",
    "protocol_type = X_train['protocol_type']\n",
    "service = X_train['service']\n",
    "flag = X_train['flag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by StandardScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-0.15544231, -0.17386919,  0.92101135, ..., -0.35267791,\n",
       "         1.98009391,  1.92949992],\n",
       "       [-0.15402019, -0.17386919, -0.92604007, ..., -0.35267791,\n",
       "        -0.60274276, -0.56532486],\n",
       "       [-0.15544231, -2.68617886, -1.30818864, ..., -0.35267791,\n",
       "        -0.60274276, -0.56532486],\n",
       "       ...,\n",
       "       [-0.15544231, -2.68617886,  0.98470278, ..., -0.35267791,\n",
       "        -0.31863073, -0.56532486],\n",
       "       [-0.15544231, -2.68617886,  1.87638277, ..., -0.35267791,\n",
       "        -0.60274276, -0.56532486],\n",
       "       [-0.15544231, -0.17386919, -0.79865721, ..., -0.35267791,\n",
       "         1.59266841,  1.92949992]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用独热编码对字符型离散变量进行处理\n",
    "le = preprocessing.LabelEncoder()\n",
    "enc = preprocessing.OneHotEncoder()\n",
    "lb = preprocessing.LabelBinarizer()\n",
    "\n",
    "# 对训练集进行 one-hot 编码\n",
    "X_train['protocol_type'] = le.fit_transform(protocol_type)\n",
    "X_train['service'] = le.fit_transform(service)\n",
    "X_train['flag'] = le.fit_transform(flag)\n",
    "labels_train = le.fit_transform(labels_train) + 1\n",
    "\n",
    "# 对测试集进行 one-hot 编码\n",
    "X_test['protocol_type'] = le.fit_transform(test_protocol_type)\n",
    "X_test['service'] = le.fit_transform(test_service)\n",
    "X_test['flag'] = le.fit_transform(test_flag)\n",
    "\n",
    "X = X_train\n",
    "standard_train_X = StandardScaler().fit_transform(X)\n",
    "standard_train_X\n",
    "\n",
    "test_X = X_test.to_numpy()\n",
    "standard_test_X = StandardScaler().fit_transform(test_X)\n",
    "standard_test_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Z-score标准化后的训练数据维度为(125972, 41)\n",
      "Z-score标准化后的测试数据维度为(22570, 41)\n"
     ]
    }
   ],
   "source": [
    "print('Z-score标准化后的训练数据维度为{0}'.format(standard_train_X.shape))\n",
    "print('Z-score标准化后的测试数据维度为{0}'.format(standard_test_X.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['back',\n",
       " 'buffer_overflow',\n",
       " 'ftp_write',\n",
       " 'guess_passwd',\n",
       " 'imap',\n",
       " 'ipsweep',\n",
       " 'land',\n",
       " 'loadmodule',\n",
       " 'multihop',\n",
       " 'neptune',\n",
       " 'nmap',\n",
       " 'normal',\n",
       " 'perl',\n",
       " 'phf',\n",
       " 'pod',\n",
       " 'portsweep',\n",
       " 'rootkit',\n",
       " 'satan',\n",
       " 'smurf',\n",
       " 'spy',\n",
       " 'teardrop',\n",
       " 'warezclient',\n",
       " 'warezmaster']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 记录下使用独热编码后编码所对应的攻击类型\n",
    "index_2_labels = []\n",
    "for i in range(1,24):\n",
    "    labels_ont_hot_index = (labels_train == i).nonzero()\n",
    "    label = labels[labels_ont_hot_index[0][0]]\n",
    "    index_2_labels.append(label)\n",
    "index_2_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "对降维后的数据进行训练用时为246.30678749084473\n",
      "精度为0.9636430990685859\n"
     ]
    }
   ],
   "source": [
    "# 该模型不是用 PCA 降维\n",
    "# 原因是使用PCA降维后的生成的模型容易将normal的行为识别为back的攻击行为\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.externals import joblib\n",
    "import time\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(standard_train_X, labels_train, test_size=.3)\n",
    "\n",
    "# clf = joblib.load('./model/NO_PCA_IDS_model.m')\n",
    "# if clf == None:\n",
    "svc = SVC(kernel='rbf', class_weight='balanced', C=0.5)\n",
    "start = time.time()\n",
    "clf = svc.fit(x_train, y_train)\n",
    "print('对降维后的数据进行训练用时为{0}'.format(time.time() - start))\n",
    "score = clf.score(x_test, y_test)\n",
    "print('精度为%s' % score)\n",
    "    # 保存模型\n",
    "#     joblib.dump(clf, './model/NO_PCA_IDS_model.m')\n",
    "#     print('save done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(Y, labels_test, evaluate_method=1):\n",
    "    acc = 0\n",
    "    TP = 0\n",
    "    FN = 0\n",
    "    FP = 0\n",
    "    TN = 0\n",
    "    if evaluate_method == 1:\n",
    "        for predict_y, real_y in zip(Y, labels_test):\n",
    "            if predict_y == 12 and real_y == 'normal':\n",
    "                acc += 1\n",
    "            if predict_y != 12 and real_y != 'normal':\n",
    "                acc += 1\n",
    "            if predict_y == 12 and real_y != 'normal':\n",
    "                FN += 1\n",
    "            if real_y == 'normal' and predict_y !=12:\n",
    "                FP += 1\n",
    "            if predict_y != 12 and real_y != 'normal':\n",
    "                TP += 1\n",
    "            if predict_y == 12 and real_y == 'normal':\n",
    "                TN += 1\n",
    "    elif evaluate_method == 2:\n",
    "        for predict_y, real_y in zip(Y, labels_test):\n",
    "            if predict_y == 12 and real_y == 12:\n",
    "                acc += 1\n",
    "            if predict_y != 12 and real_y != 12:\n",
    "                acc += 1\n",
    "            if predict_y == 12 and real_y != 12:\n",
    "                FN += 1\n",
    "            if real_y == 12 and predict_y !=12:\n",
    "                FP += 1\n",
    "            if predict_y != 12 and real_y != 12:\n",
    "                TP += 1\n",
    "            if predict_y == 12 and real_y == 12:\n",
    "                TN += 1\n",
    "\n",
    "    precent_acc = acc / len(labels_test)\n",
    "    print(TP, FN)\n",
    "    print(FP, TN)\n",
    "    TPR = TP / (TP + FN)\n",
    "    FPR = FP / (FP + TN)\n",
    "    print('acc radio %s ' % precent_acc)\n",
    "    print('TPR is %s' % TPR)\n",
    "    print('FPR is %s' % FPR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9147 3692\n",
      "410 9321\n",
      "acc radio 0.8182543198936642 \n",
      "TPR is 0.712438663447309\n",
      "FPR is 0.042133388140992704\n"
     ]
    }
   ],
   "source": [
    "# 对 NSL-KDD-test_set 进行模型评估\n",
    "Y = clf.predict(standard_test_X)\n",
    "evaluate(Y, labels_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17484 34\n",
      "1286 18988\n",
      "acc radio 0.9650719729043183 \n",
      "TPR is 0.9980591391711383\n",
      "FPR is 0.06343099536351977\n"
     ]
    }
   ],
   "source": [
    "# 对 NSL-KDD-30%-train_set 进行模型评估\n",
    "Y = clf.predict(x_test)\n",
    "evaluate(Y, y_test, evaluate_method=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_labels = {'DOS':['back','neptune','smurf','teardrop','land','pod','apache2','mailbomd','processtable'],\n",
    "              'Probe':['satan','portsweep','ipsweep','nmap','mscan','saint'],\n",
    "              'R2L':['warezmaster','ftp_write','guess_passwd','imap','multihop','phf','spy','warezclient','sendmail','named','snmpgetattack','snmpguess','xlock','xsnoop','worm'],\n",
    "              'U2R':['rootkit','buffer_overflow','loadmodule','perl','httptunnel','ps','sqlattack','xterm'],\n",
    "              'NORMAL':['normal']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['DOS', 'Probe', 'R2L', 'U2R', 'NORMAL'])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "types_attack = all_labels.keys()\n",
    "types_attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_result = {'target_types':[],'predict_types':[]}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
